Imports

! rm -rf Datasets

# Standard Imports
import os

# Third-Party Imports
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.utils import image_dataset_from_directory
import keras_tuner as kt

import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import plotly.figure_factory as ff
import plotly.offline as pyo
pyo.init_notebook_mode()

import numpy as np
import pandas as pd

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

import splitfolders

Requirement already satisfied: tensorflow-addons in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (0.19.0)
Requirement already satisfied: typeguard>=2.7 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from tensorflow-addons) (3.0.1)
Requirement already satisfied: packaging in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from tensorflow-addons) (23.0)
Requirement already satisfied: typing-extensions>=4.4.0 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from typeguard>=2.7->tensorflow-addons) (4.5.0)
Requirement already satisfied: importlib-metadata>=3.6 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from typeguard>=2.7->tensorflow-addons) (6.1.0)
Requirement already satisfied: zipp>=0.5 in /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages (from importlib-metadata>=3.6->typeguard>=2.7->tensorflow-addons) (3.15.0)

Utility Functions

Plotting

# Plotting functions
def plot_data(x=None, y=None, z=None, title="", x_label="", y_label="", name="", mode="markers", text="", **traces):
    fig = go.Figure(layout={
        "title": title,
        "xaxis": {"title": x_label},
        "yaxis": {"title": y_label}
    })
    
    if z is None:
        data = go.Scatter(
            x=x,
            y=y,
            mode=mode,
            name=name,
            text=text
        )
    else:
        data = go.Scatter3d(
            x=x,
            y=y,
            z=z,
            mode=mode,
            name=name,
            text=text
        )

    if x is not None and y is not None:
        fig.add_trace(data)
    
    for t in traces:
        fig.add_trace(traces[t])
    
    return fig

def plot_bar_data(*bars, x=None, title="", x_label="", y_label=""):
    fig = go.Figure(
        layout={
            "title": title,
            "xaxis": {"title": x_label},
            "yaxis": {"title": y_label},
            "barmode": "group"
        }, data=[
            go.Bar(name=f"{bar[0]}", x=x, y=bar[1])
            for bar in bars
        ])
    
    return fig
    
def create_trace(x=None, y=None, z=None, name="", mode="lines", text="", marker_size=None):
    if z is None:
        trace = go.Scatter(
            x=x,
            y=y,
            mode=mode,
            name=name,
            text=text,
            marker=dict(size=marker_size)
        )
    else:
        trace = go.Scatter3d(
            x=x,
            y=y,
            z=z,
            mode=mode,
            name=name,
            text=text,
            marker=dict(size=marker_size)
        )
    
    return trace

def create_confusion_matrix(dataset, preds):
    preds = 1 * (preds >= 0.5).flatten()
    labels = []
    
    for x, y in dataset.map(lambda x, y: (x, y)):
        labels.append(y.numpy())    
    labels = np.concatenate(labels)
    
    basic_cm = confusion_matrix(labels, preds)
    return basic_cm

def plot_confusion_matrix(cm, x=[0,1], y=[0,1], title="", x_label="Predicted Label", y_label="True Label"):
    fig = ff.create_annotated_heatmap(
        cm, 
        x=x, 
        y=y, 
        annotation_text=cm, 
        colorscale='Viridis',
        showscale=True,
    )
    
    fig.update_layout({
        "title": title,
        "xaxis": {"title": x_label, "side": "bottom"},
        "yaxis": {"title": y_label, "autorange": "reversed"},
        "coloraxis": { "colorbar": {"orientation": "h"}}
    })
    
    return fig

def plot_confusion_matrix_2(cm, display_labels=["Evil", "Good"]):
    return ConfusionMatrixDisplay(
        confusion_matrix=cm, display_labels=display_labels
    )
    
    return cm

def plot_metric_table(metrics):
    # Initialise figure
    fig = go.Figure()
    
    # Create table
    headers = dict(values=["", "Training Set", "Validation Set", "Test Set"])
    first_column = [
        "F1 Score",
        "Accuracy",
        "Precision", 
        "Recall", 
        "True Postives",
        "False Positives",
        "True Negatives",
        "False Negatives",
        "Loss"
    ]
    
    cells = [first_column] + [[
        round(metrics[ds]["f1_score"], 4),
        round(metrics[ds]["accuracy"], 4),
        round(metrics[ds]["precision"], 4),
        round(metrics[ds]["recall"], 4),
        metrics[ds]["tp"],
        metrics[ds]["fp"],
        metrics[ds]["tn"],
        metrics[ds]["fn"],
        round(metrics[ds]["loss"], 4)
    ] for ds in ("train_set", "val_set", "test_set")]
    
    data = go.Table(
        header=headers,
        cells=dict(values=cells)
    )
    
    fig.add_trace(data)
    return fig

def train_plot_collection(training_plots, metric_table, title=""):
    # Initialise make_subplots object
    fig = make_subplots(
        rows=4, 
        cols=2, 
        subplot_titles=[
            "Accuracy", 
            "Loss",
            "Precision",
            "Recall",
            "F1_score",
            "Key Metrics",
        ], 
        specs=[
            [{"type": "xy"}, {"type": "xy"}],
            [{"type": "xy"}, {"type": "xy"}],
            [{"type": "xy", "colspan": 2}, None],
            [{"type": "table", "colspan": 2}, None],
        ]
    )
    
    fig.update_layout({
        "title": title,
        "height": 1500,
    })
    
    # Add traces
    for i in range(len(training_plots["accuracy"].data)):
        fig.add_trace(training_plots["accuracy"].data[i], row=1, col=1)
    
    for i in range(len(training_plots["loss"].data)):
        fig.add_trace(training_plots["loss"].data[i], row=1, col=2)
        
    for i in range(len(training_plots["precision"].data)):
        fig.add_trace(training_plots["precision"].data[i], row=2, col=1)
        
    for i in range(len(training_plots["recall"].data)):
        fig.add_trace(training_plots["recall"].data[i], row=2, col=2)
      
    for i in range(len(training_plots["f1_score"].data)):
        fig.add_trace(training_plots["f1_score"].data[i], row=3, col=1)
        
    fig.add_trace(metric_table.data[0], row=4, col=1)
    
    # Update axes
    fig.update_xaxes(title_text="Epochs", row=1, col=1)
    fig.update_xaxes(title_text="Epochs", row=1, col=2)
    fig.update_xaxes(title_text="Epochs", row=2, col=1)
    fig.update_xaxes(title_text="Epochs", row=2, col=2)
    fig.update_xaxes(title_text="Epochs", row=3, col=1)
    
    return fig

def test_plot_collection(metric_plot, true_false_plot, title=""):
    # Initialise make_subplots object
    fig = make_subplots(
        rows=2,
        cols=1,
        shared_xaxes=True,
        vertical_spacing=0.03,
    )
    
    fig.update_layout({
        "title": title,
        "height": 1000,
    })
    
    # Add Traces
    for i in range(len(metric_plot.data)):
        fig.add_trace(metric_plot.data[i], row=1, col=1)
    for i in range(len(true_false_plot.data)):
        fig.add_trace(true_false_plot.data[i], row=2, col=1)
    
    return fig

Loading Data

# Global variables
train_dir = "Datasets/train" # repo for training set
val_dir = "Datasets/val" # repo for validation set
test_dir = "Datasets/test" # repo for test set

img_h = 140 # image height
img_w = 140 # image width
batch_size = 32

label_map = {0: "evil", 1: "good"} # "evil" is the negative class; "good" is the positive class
input_shape = (img_h, img_w, 3)
dataset_split_ratio = (0.7, 0.15, 0.15)

seed = 312

# Splitting dataset into training, validation and testing sets
splitfolders.ratio(
    "Images", 
    output="Datasets", 
    ratio=dataset_split_ratio,
    seed=seed,
)
# source: https://github.com/jfilter/split-folders

Copying files: 597 files [00:00, 1169.27 files/s]

# Load training set
train_ds = image_dataset_from_directory(
    train_dir,
    validation_split=0,
    seed=seed,
    image_size=(img_h, img_w),
    batch_size=batch_size
)

# Load validation set
val_ds = image_dataset_from_directory(
    val_dir, 
    validation_split=0,
    seed=seed,
    image_size=(img_h, img_w),
    batch_size=batch_size
)

# Load testing set
test_ds = image_dataset_from_directory(
    test_dir, 
    validation_split=0,
    seed=seed,
    image_size=(img_h, img_w),
    batch_size=batch_size
)

Found 417 files belonging to 2 classes.
Found 89 files belonging to 2 classes.
Found 91 files belonging to 2 classes.

Getting Class Weights

y1 = np.concatenate([y for x, y in train_ds], axis=0)
y2 = np.concatenate([y for x, y in val_ds], axis=0)
y3 = np.concatenate([y for x, y in test_ds], axis=0)

class_totals = np.bincount(y1) + np.bincount(y2) + np.bincount(y3)
neg, pos = class_totals[0], class_totals[1]
total = neg + pos

evil_weight = (1 / neg) * (total / 2.0)
good_weight = (1 / pos) * (total / 2.0)

class_weights = {0: evil_weight, 1: good_weight}
class_weights

# source: https://www.tensorflow.org/tutorials/structured_data/imbalanced_data#calculate_class_weights

{0: 1.515228426395939, 1: 0.74625}

Example Images

for images, labels in train_ds.take(1):
    for i in range(batch_size // 16):
        print(label_map[labels[i].numpy()])
        go.Figure(go.Image(z=images[i].numpy())).show()   

good

evil

Model Implementations

Basic Models

Perceptron

def perceptron(input_shape, num_outputs=1):
    model = tf.keras.Sequential([
        tf.keras.layers.Rescaling(1./255),
        tf.keras.layers.Flatten(input_shape=input_shape),
        tf.keras.layers.Dense(num_outputs, activation='sigmoid')
    ])
    
    return model

Test Model 1 (Single Convolutional Layer)

def test_cnn_1(input_shape, num_outputs=1):
    model = tf.keras.Sequential([
        tf.keras.layers.Rescaling(1./255),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Flatten(),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dense(num_outputs, activation="sigmoid")
    ])

    return model

# source: https://www.tensorflow.org/tutorials/images/cnn

Test Model 2 (Two Covolutional Layers)

def test_cnn_2(input_shape, num_outputs=1):
    model = tf.keras.Sequential([
        tf.keras.layers.Rescaling(1./255),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Conv2D(32, (3, 3), activation="relu", kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Flatten(),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dense(num_outputs, activation='sigmoid')
    ])

    return model

# source: https://www.tensorflow.org/tutorials/images/cnn

Test Model 3 (Three Convolutional Layers)

def test_cnn_3(input_shape, num_outputs=1):
    model = tf.keras.Sequential([
        tf.keras.layers.Rescaling(1./255),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Conv2D(16, (3, 3), activation="relu", input_shape=input_shape, kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Conv2D(32, (3, 3), activation="relu", kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Conv2D(64, (3, 3), activation="relu", kernel_regularizer="l2"),
        tf.keras.layers.MaxPooling2D((2, 2)),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Flatten(),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dense(1, activation="sigmoid")
    ])

    return model

# source: https://www.tensorflow.org/tutorials/images/cnn

Model Pipeline

Helper Functions

def get_model(model_name, input_shape):
    models = {
        "perceptron": perceptron,
        "test_cnn_1": test_cnn_1,
        "test_cnn_2": test_cnn_2,
        "test_cnn_3": test_cnn_3,
    }
    
    return models[model_name](input_shape)

def get_optimiser(opt_name, l_rate, epsilon):
    optimisers = {
        "adam": tf.keras.optimizers.Adam,
        "sgd": tf.keras.optimizers.experimental.SGD,
        "adadelta": tf.keras.optimizers.experimental.Adadelta,
        "adagrad": tf.keras.optimizers.experimental.Adagrad,
        "adamax": tf.keras.optimizers.experimental.Adamax,
        "nadam": tf.keras.optimizers.experimental.Nadam,
        "rmsprop": tf.keras.optimizers.experimental.RMSprop
    }
    
    if opt_name == "sgd": return optimisers[opt_name](learning_rate=l_rate)
    return optimisers[opt_name](learning_rate=l_rate, epsilon=epsilon)

def get_train_performance_plots(history, epochs, loss_metrics, acc_metrics, prec_metrics, rec_metrics, f1_metrics):
    x = np.arange(1, epochs + 1)
    
    loss_traces = {
        m: create_trace(x, history[m], name=m) 
        for m in loss_metrics
    }

    f1_traces = {
        m: create_trace(x, history[m], name=m) 
        for m in f1_metrics
    }

    accuracy_traces = {
        m: create_trace(x, history[m], name=m) 
        for m in acc_metrics
    }
    
    precision_traces = {
        m: create_trace(x, history[m], name=m) 
        for m in prec_metrics
    }
    
    recall_traces = {
        m: create_trace(x, history[m], name=m) 
        for m in rec_metrics
    }
    
    train_acc_plot = plot_data(
        title="Training Accuracy", 
        x_label="Epochs", 
        y_label="Accuracy", 
        **accuracy_traces
    )
    
    train_loss_plot = plot_data(
        title="Training Loss", 
        x_label="Epochs", 
        y_label="Loss", 
        **loss_traces
    )
    
    train_prec_plot = plot_data(
        title="Training Precision", 
        x_label="Epochs", 
        y_label="Precision", 
        **precision_traces
    )
    
    train_rec_plot = plot_data(
        title="Training Recall", 
        x_label="Epochs", 
        y_label="Recall", 
        **recall_traces
    )

    train_f1_plot = plot_data(
        title="Training F1 Score", 
        x_label="Epochs", 
        y_label="F1 Score", 
        **f1_traces
    )
    
    return train_acc_plot, train_loss_plot, train_prec_plot, train_rec_plot, train_f1_plot

def get_confusion_matrix(cm):
    return plot_confusion_matrix(cm, x=["Evil", "Good"], y=["Evil", "Good"])

def get_confusion_matrix_2(cm, display_labels=["Evil", "Good"]):
    return plot_confusion_matrix_2(cm, display_labels)

def get_metric_table(metrics):
    return plot_metric_table(metrics)

Pipeline Function

def run_model(
    model_name, 
    train_set, 
    val_set, 
    test_set, 
    input_shape=(140,140,3), 
    epochs=10, 
    opt_name="sgd", 
    l_rate=0.001, 
    epsilon=1e-7, 
    class_weight=None,
    verbosity="auto",
):
    # Choose and initialise model
    model = get_model(model_name, input_shape)
    
    # Choose and initialise optimiser
    optimiser = get_optimiser(opt_name, l_rate, epsilon)
    
    # Compile model
    model.compile(
        optimizer=optimiser,
        loss="binary_crossentropy",
        metrics=[
            tfa.metrics.F1Score(
                num_classes=2,
                threshold=0.5,
                average="micro"
            ),
            "accuracy",
            tf.keras.metrics.Precision(),
            tf.keras.metrics.Recall(),
        ]
    )
    
    # Fit model to data
    history = model.fit(
        train_set, 
        epochs=epochs,
        validation_data=val_set,
        shuffle=True,
        class_weight=class_weight,
        verbose=verbosity,
    ).history
    
    keys = list(history.keys())
    
    for k in keys:
        if "val_accuracy" in k and k != "val_accuracy":
            history["val_precision"] = history[k]
        elif "val_f1_score" in k and k != "val_f1_score":
            history["val_recall"] = history[k]
        elif "accuracy" in k and k != "accuracy":
            history["precision"] = history[k]
        elif "f1_score" in k and k != "f1_score":
            history["recall"] = history[k]
        elif "val_precision" in k and k != "val_precision":
            history["val_precision"] = history[k]
        elif "val_recall" in k and k != "val_recall":
            history["val_recall"] = history[k]
        elif "precision" in k and k != "precision":
            history["precision"] = history[k]
        elif "recall" in k and k != "recall":
            history["recall"] = history[k]
        
    
    # Evaluate model on train, validation and test sets
    metric_names = ["loss", "f1_score", "accuracy", "precision", "recall"]
    
    train_metrics = model.evaluate(train_set)
    val_metrics = model.evaluate(val_set)
    test_metrics = model.evaluate(test_set)
    
    metrics = {
        "train_set": { 
            name: metric 
            for name, metric in zip(metric_names, train_metrics) 
        },
        "val_set": { 
            name: metric 
            for name, metric in zip(metric_names, val_metrics) 
        },
        "test_set": { 
            name: metric 
            for name, metric in zip(metric_names, test_metrics) 
        },
    }
    
    # Generate predictions on the test set
    preds = {
        "train_set": model.predict(train_set),
        "val_set": model.predict(val_set),
        "test_set": model.predict(test_set),
    }
        
    # Build training and loss graphs
    loss_metrics = ["loss", "val_loss"]
    accuracy_metrics = ["accuracy", "val_accuracy"]
    prec_metrics = ["precision", "val_precision"]
    rec_metrics = ["recall", "val_recall"]
    f1_metrics = ["f1_score", "val_f1_score"]
    
    train_acc_plot, train_loss_plot, train_prec_plot, train_rec_plot, train_f1_plot = get_train_performance_plots(
        history, 
        epochs, 
        loss_metrics, 
        accuracy_metrics,
        prec_metrics,
        rec_metrics,
        f1_metrics,
    )
    
    training_plots = {
        "accuracy": train_acc_plot,
        "loss": train_loss_plot,
        "precision": train_prec_plot,
        "recall": train_rec_plot,
        "f1_score": train_f1_plot
    }
    
    # Get cofusion matrices for train, validation and test sets
    basic_cms = {
        "train_set": create_confusion_matrix(train_set, preds["train_set"]),
        "val_set": create_confusion_matrix(val_set, preds["val_set"]),
        "test_set": create_confusion_matrix(test_set, preds["test_set"]),
    }
    
    confusion_matrices = {
        "train_set": get_confusion_matrix(basic_cms["train_set"]),
        "val_set": get_confusion_matrix(basic_cms["val_set"]),
        "test_set": get_confusion_matrix(basic_cms["test_set"])
    }
    
    # Append new metrics from confusion matrices
    for d_set in ("train_set", "val_set", "test_set"):
        metrics[d_set]["tp"] = basic_cms[d_set][1][1]
        metrics[d_set]["fp"] = basic_cms[d_set][0][1]
        metrics[d_set]["fn"] = basic_cms[d_set][1][0]
        metrics[d_set]["tn"] = basic_cms[d_set][0][0]
    
    # Create metric table
    metric_table = get_metric_table(metrics)
        
    return model, metrics, metric_table, preds, training_plots, basic_cms, confusion_matrices

Example Model

model, metrics, metric_table, preds, training_plots, basic_cms, confusion_matrices = run_model(
    model_name="test_cnn_1", 
    train_set=train_ds, 
    val_set=val_ds, 
    test_set=test_ds,
    input_shape=input_shape,
    epochs=10,
    l_rate=0.01,
    opt_name="sgd",
    class_weight=class_weights,
)

Epoch 1/10
WARNING:tensorflow:From /Users/bhekimaenetja/.local/share/virtualenvs/small-projects-ai-NRjJWIjk/lib/python3.9/site-packages/tensorflow/python/autograph/pyct/static_analysis/liveness.py:83: Analyzer.lamba_check (from tensorflow.python.autograph.pyct.static_analysis.liveness) is deprecated and will be removed after 2023-09-23.
Instructions for updating:
Lambda fuctions will be no more assumed to be used in the statement where they are used, or at least in the same block. https://github.com/tensorflow/tensorflow/issues/56089
14/14 [==============================] - 5s 250ms/step - loss: 1.5125 - f1_score: 0.5904 - accuracy: 0.5108 - precision: 0.6743 - recall: 0.5250 - val_loss: 0.7193 - val_f1_score: 0.7973 - val_accuracy: 0.6629 - val_precision: 0.6705 - val_recall: 0.9833
Epoch 2/10
14/14 [==============================] - 3s 231ms/step - loss: 0.9615 - f1_score: 0.6937 - accuracy: 0.6379 - precision: 0.8028 - recall: 0.6107 - val_loss: 0.8414 - val_f1_score: 0.8108 - val_accuracy: 0.6854 - val_precision: 0.6818 - val_recall: 1.0000
Epoch 3/10
14/14 [==============================] - 3s 216ms/step - loss: 0.9126 - f1_score: 0.7076 - accuracy: 0.6571 - precision: 0.8278 - recall: 0.6179 - val_loss: 1.0567 - val_f1_score: 0.5714 - val_accuracy: 0.4944 - val_precision: 0.6667 - val_recall: 0.5000
Epoch 4/10
14/14 [==============================] - 3s 227ms/step - loss: 0.6126 - f1_score: 0.7960 - accuracy: 0.7554 - precision: 0.9045 - recall: 0.7107 - val_loss: 0.6394 - val_f1_score: 0.8088 - val_accuracy: 0.7079 - val_precision: 0.7237 - val_recall: 0.9167
Epoch 5/10
14/14 [==============================] - 3s 230ms/step - loss: 0.5736 - f1_score: 0.7921 - accuracy: 0.7482 - precision: 0.8889 - recall: 0.7143 - val_loss: 0.7195 - val_f1_score: 0.7727 - val_accuracy: 0.6629 - val_precision: 0.7083 - val_recall: 0.8500
Epoch 6/10
14/14 [==============================] - 3s 234ms/step - loss: 0.3942 - f1_score: 0.8413 - accuracy: 0.8082 - precision: 0.9464 - recall: 0.7571 - val_loss: 0.7452 - val_f1_score: 0.7538 - val_accuracy: 0.6404 - val_precision: 0.7000 - val_recall: 0.8167
Epoch 7/10
14/14 [==============================] - 3s 232ms/step - loss: 0.3944 - f1_score: 0.8599 - accuracy: 0.8273 - precision: 0.9444 - recall: 0.7893 - val_loss: 0.7594 - val_f1_score: 0.7360 - val_accuracy: 0.6292 - val_precision: 0.7077 - val_recall: 0.7667
Epoch 8/10
14/14 [==============================] - 3s 224ms/step - loss: 0.4234 - f1_score: 0.8571 - accuracy: 0.8249 - precision: 0.9481 - recall: 0.7821 - val_loss: 0.8510 - val_f1_score: 0.7143 - val_accuracy: 0.5955 - val_precision: 0.6818 - val_recall: 0.7500
Epoch 9/10
14/14 [==============================] - 3s 227ms/step - loss: 0.4464 - f1_score: 0.8642 - accuracy: 0.8297 - precision: 0.9300 - recall: 0.8071 - val_loss: 0.8541 - val_f1_score: 0.7500 - val_accuracy: 0.6404 - val_precision: 0.7059 - val_recall: 0.8000
Epoch 10/10
14/14 [==============================] - 3s 218ms/step - loss: 0.3185 - f1_score: 0.9015 - accuracy: 0.8753 - precision: 0.9597 - recall: 0.8500 - val_loss: 0.8353 - val_f1_score: 0.7692 - val_accuracy: 0.6629 - val_precision: 0.7143 - val_recall: 0.8333
14/14 [==============================] - 1s 38ms/step - loss: 0.2060 - f1_score: 0.9704 - accuracy: 0.9592 - precision: 0.9458 - recall: 0.9964
3/3 [==============================] - 0s 42ms/step - loss: 0.8353 - f1_score: 0.7692 - accuracy: 0.6629 - precision: 0.7143 - recall: 0.8333
3/3 [==============================] - 0s 40ms/step - loss: 0.8737 - f1_score: 0.7500 - accuracy: 0.6264 - precision: 0.6711 - recall: 0.8500
14/14 [==============================] - 1s 39ms/step
3/3 [==============================] - 0s 37ms/step
3/3 [==============================] - 0s 68ms/step

train_plot_collection(training_plots, metric_table, "Training Results for Test Model 1")

# Get predictions
preds["test_set"]

array([[0.81288254],
       [0.611088  ],
       [0.8975335 ],
       [0.7974999 ],
       [0.8789054 ],
       [0.95714366],
       [0.8393403 ],
       [0.89898956],
       [0.03007286],
       [0.2317079 ],
       [0.9409177 ],
       [0.31317553],
       [0.84566253],
       [0.8655793 ],
       [0.94851184],
       [0.88803065],
       [0.90942335],
       [0.9584492 ],
       [0.96664816],
       [0.7384149 ],
       [0.57421404],
       [0.9040143 ],
       [0.30616298],
       [0.8886775 ],
       [0.96876436],
       [0.99736655],
       [0.8795997 ],
       [0.9006875 ],
       [0.7856223 ],
       [0.47414273],
       [0.96970284],
       [0.84734094],
       [0.4006444 ],
       [0.9896213 ],
       [0.88879615],
       [0.1733048 ],
       [0.8556108 ],
       [0.7576365 ],
       [0.9480191 ],
       [0.9472297 ],
       [0.58313245],
       [0.83794457],
       [0.98230207],
       [0.6320949 ],
       [0.38295907],
       [0.7878443 ],
       [0.7271949 ],
       [0.9726155 ],
       [0.7567242 ],
       [0.98197377],
       [0.86900985],
       [0.88539845],
       [0.47246763],
       [0.6321067 ],
       [0.3614071 ],
       [0.8880345 ],
       [0.7326922 ],
       [0.85527843],
       [0.02947607],
       [0.87678087],
       [0.9179377 ],
       [0.8151784 ],
       [0.64918774],
       [0.94750154],
       [0.7668669 ],
       [0.9664161 ],
       [0.74536943],
       [0.2750769 ],
       [0.5977465 ],
       [0.43228367],
       [0.9477465 ],
       [0.92996174],
       [0.81136954],
       [0.671169  ],
       [0.73293364],
       [0.7469404 ],
       [0.74196726],
       [0.6985409 ],
       [0.69045407],
       [0.795322  ],
       [0.9368606 ],
       [0.59249955],
       [0.96143323],
       [0.809855  ],
       [0.9753247 ],
       [0.7798728 ],
       [0.85444766],
       [0.24379267],
       [0.6337953 ],
       [0.95332026],
       [0.4746781 ]], dtype=float32)

# Confusion matrix to tell us the number of true positives, false negatives etc.
confusion_matrices["train_set"]

confusion_matrices["val_set"]

confusion_matrices["test_set"]

Finding the Best Model

In order to find the best model for this classification task we'll follow a simple 3-step process:

1. For each model we'll see how its performance varies when the choice of certain hyperparameters are adjusted through a range of values; we'll adjust the hyperaparameters using the keras_tuner library.
2. We will then adjust the number of epochs in training in order to determine the point when the model begins to coverge on the training and validation sets.
3. The models (with their newly tuned hyperparameters) will then be compared against each other to decide what is the best architecture for the classification task.

The hyperparameters that we'll look at are: # of epochs, learning rate, optimiser and epsilon.

Helper Functions

def get_test_plots(x, acc, prec, rec, f1, tp, tn, fp, fn):
    true_false_plot = plot_bar_data(
        tp,
        tn,
        fp,
        fn,
        x=x,
        x_label="Epoch Number",
    )
    
    metric_plot = plot_bar_data(
        acc,
        prec,
        rec,
        f1,
        x=x,
        title="Test Bar Chart 2",
        x_label="Parameter",
    )
    
    return true_false_plot, metric_plot

Hyperparameter Tuning Functions

def tune_hyper_params(model_name, optimiser_range):
    def model_builder(hp):
        model = get_model(model_name, input_shape=(140, 140, 3))

        hp_l_rate = hp.Float("learning_rate", min_value=0, max_value=1, step=1e-08)
        hp_opt_name = hp.Choice("optimiser", values=optimiser_range)
        hp_epsilon = hp.Float("epsilon", min_value=0, max_value=1, step=1e-08)

        optimiser = get_optimiser(hp_opt_name, hp_l_rate, hp_epsilon)
        model.compile(
            optimizer=optimiser,
            loss="binary_crossentropy",
            metrics=[
                "accuracy",
                tf.keras.metrics.Precision(),
                tf.keras.metrics.Recall(),
            ]
        )
        
        return model
    return model_builder

def get_tuner(model_builder, tuner_name):
    if tuner_name == "bayesian":
        tuner = kt.BayesianOptimization(
            model_builder,
            max_trials=5,
            objective="val_accuracy",
            directory="",
            overwrite=True,
        )
    elif tuner_name == "random":
        tuner = kt.RandomSearch(
            model_builder,
            max_trials=5,
            objective="val_accuracy",
            directory="",
            overwrite=True,
        )
    
    return tuner

def get_best_hyperparams(model_name, train_set, val_set, optimiser_range, tuner_name, class_weights=class_weights):
    print(f"Class weight is: {class_weights}")
    model_builder = tune_hyper_params(model_name, optimiser_range)
    tuner = get_tuner(model_builder, tuner_name)

    tuner.search(
        train_set, 
        epochs=5,
        validation_data=val_set,
        shuffle=True,
        class_weight=class_weights,
        verbose=1,
    )

    best_hps = tuner.get_best_hyperparameters(num_trials=50)[0]
    
    return {
        "optimiser": best_hps.get("optimiser"),
        "l_rate": best_hps.get("learning_rate"),
        "epsilon": best_hps.get("epsilon"),
    }

# source: https://keras.io/api/keras_tuner/

Comparison Function

def compare_configs(test_params, model_name, train_set, val_set, test_set, control_params={}):
    # Check if all param ranges are of equal length
    vals = list(test_params.values())
    val_len = len(vals[0])
    for v in vals[1:]:
        if len(v) != val_len:
            raise Exception("Error: Param ranges are not all the same!")
    
    # Initialise parameters for running model
    default_params = {
        "model_name": model_name,
        "train_set": train_set,
        "val_set": val_set,
        "test_set": test_set,
        "epochs": 10,
        "opt_name": "sgd",
        "l_rate": 0.001,
        "epsilon": 1e-7,
        "class_weight": None,
        "verbosity": 0,
    }
    
    default_params.update(control_params)
    
    # Initialise X-axis
    param_names = list(test_params.keys())
    x = []
    
    for i in range(val_len):
        x_string = ""
        for n in param_names:
            x_string += f" {n} = {test_params[n][i]};"
        x.append(x_string[:-1].strip())
        
    # Iterate through parameters
    acc = ("Accuracy", [])
    prec = ("Precision", [])
    rec = ("Recall", [])
    f1 = ("F1 Score", [])
    
    tp = ("True Positives", [])
    tn = ("True Negatives", [])
    fp = ("False Posivites", [])
    fn = ("False Negatives", [])
    
    for i in range(val_len):
        params_copy = default_params.copy()
        test_params_i = {n: test_params[n][i] for n in param_names}
        params_copy.update(test_params_i)
        
        model, metrics, metric_table, _, training_plots, _, _ = run_model(**params_copy)
        train_results = train_plot_collection(
            training_plots, 
            metric_table, 
            f"Results ({x[i]})"
        )
        
        acc[1].append(metrics["test_set"]["accuracy"])
        prec[1].append(metrics["test_set"]["precision"])
        rec[1].append(metrics["test_set"]["recall"])
        f1[1].append(metrics["test_set"]["f1_score"])
        tp[1].append(metrics["test_set"]["tp"])
        tn[1].append(metrics["test_set"]["tn"])
        fp[1].append(metrics["test_set"]["fp"])
        fn[1].append(metrics["test_set"]["fn"])
        
        train_results.show()
    
    # Get final result plots
    true_false_plot, metric_plot = get_test_plots(x,acc,prec,rec,f1,tp,tn,fp,fn)
    
    results = test_plot_collection(
        true_false_plot, 
        metric_plot, 
        f"Test set results when adjusting: {', '.join([n for n in param_names])}"
    )
    
    return results

Intial Hyperparameter Ranges

optimiser_range = ["sgd", "adam", "adadelta", "adagrad", "adamax", "nadam", "rmsprop"]
hp_search_methods = ["random", "bayesian"]

Testing

model_hps = dict()

Perceptron

model_name = "perceptron"

Step 1: Use a variety of search methods to determine the best learning rate, epsilon and optimiser

Calculating the best hyperparameters using random, hyperband and bayesian search

best_hps = {
    method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
    for method in hp_search_methods
}
best_hps

Trial 5 Complete [00h 00m 02s]
val_accuracy: 0.6741573214530945

Best val_accuracy So Far: 0.6741573214530945
Total elapsed time: 00h 00m 12s
INFO:tensorflow:Oracle triggered exit

{'random': {'optimiser': 'rmsprop',
  'l_rate': 0.62089814,
  'epsilon': 0.61781427},
 'bayesian': {'optimiser': 'adagrad',
  'l_rate': 0.022246600000000002,
  'epsilon': 0.7611846600000001}}

Comparing model performace with the hyperparameters yielded by the different search methods

# parameters that we want to iterate through
hp_matrix = [
    x
    for x in set(
        (
            best_hps[hp]["optimiser"], 
            best_hps[hp]["l_rate"],
            best_hps[hp]["epsilon"],
        ) 
        for hp in hp_search_methods
    )
]

test_params = { 
    "opt_name": [hp[0] for hp in hp_matrix],
    "l_rate": [hp[1] for hp in hp_matrix],
    "epsilon": [hp[2] for hp in hp_matrix],
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
control_params = {
    "model_name": model_name,
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 0s 5ms/step - loss: 27.4598 - f1_score: 0.8034 - accuracy: 0.6715 - precision_1: 0.6715 - recall_1: 1.0000
3/3 [==============================] - 0s 7ms/step - loss: 25.7476 - f1_score: 0.8054 - accuracy: 0.6742 - precision_1: 0.6742 - recall_1: 1.0000
3/3 [==============================] - 0s 7ms/step - loss: 30.5270 - f1_score: 0.7947 - accuracy: 0.6593 - precision_1: 0.6593 - recall_1: 1.0000
14/14 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 4ms/step
3/3 [==============================] - 0s 5ms/step

14/14 [==============================] - 0s 6ms/step - loss: 663.3700 - f1_score: 0.8046 - accuracy: 0.6739 - precision_2: 0.6731 - recall_2: 1.0000
3/3 [==============================] - 0s 7ms/step - loss: 709.2269 - f1_score: 0.8054 - accuracy: 0.6742 - precision_2: 0.6742 - recall_2: 1.0000
3/3 [==============================] - 0s 8ms/step - loss: 905.7441 - f1_score: 0.7947 - accuracy: 0.6593 - precision_2: 0.6593 - recall_2: 1.0000
14/14 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 4ms/step

Best hyperparameters:

• Optimiser —> adagrad
• Learning rate —> 0.022246600000000002
• Epsilon —> 0.7611846600000001

best_optimiser = "adagrad"
best_l_rate =  0.022246600000000002
best_epsilon =  0.7611846600000001

Step 2: Adjusting epochs with newly tuned hyperparameters

# parameters that we want to iterate through
test_params = { 
    "epochs": [50]
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
    "model_name": model_name,
    "opt_name": best_optimiser,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon,
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 0s 5ms/step - loss: 0.0410 - f1_score: 0.9928 - accuracy: 0.9904 - precision_3: 1.0000 - recall_3: 0.9857
3/3 [==============================] - 0s 6ms/step - loss: 2.1981 - f1_score: 0.6111 - accuracy: 0.5281 - precision_3: 0.6875 - recall_3: 0.5500
3/3 [==============================] - 0s 7ms/step - loss: 2.1148 - f1_score: 0.6949 - accuracy: 0.6044 - precision_3: 0.7069 - recall_3: 0.6833
14/14 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 4ms/step
3/3 [==============================] - 0s 4ms/step

Ideal epoch number: 20

model_hps[model_name] = {
    "epochs": 20,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon,
    "optimiser": best_optimiser
}
model_hps

{'perceptron': {'epochs': 20,
  'l_rate': 0.022246600000000002,
  'epsilon': 0.7611846600000001,
  'optimiser': 'adagrad'}}

Test Model 1

model_name = "test_cnn_1"

Step 1: Use a variety of search methods to determine the best learning rate, epsilon and optimiser

Calculating the best hyperparameters using random, hyperband and bayesian search

best_hps = {
    method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
    for method in hp_search_methods
}
best_hps

Trial 5 Complete [00h 00m 18s]
val_accuracy: 0.6853932738304138

Best val_accuracy So Far: 0.7078651785850525
Total elapsed time: 00h 01m 30s
INFO:tensorflow:Oracle triggered exit

{'random': {'optimiser': 'adam', 'l_rate': 0.50438527, 'epsilon': 0.05359655},
 'bayesian': {'optimiser': 'adagrad',
  'l_rate': 0.52687047,
  'epsilon': 0.89892038}}

Comparing model performace with the hyperparameters yielded by the different search methods

# parameters that we want to iterate through
hp_matrix = [
    x
    for x in set(
        (
            best_hps[hp]["optimiser"], 
            best_hps[hp]["l_rate"],
            best_hps[hp]["epsilon"],
        ) 
        for hp in hp_search_methods
    )
]

test_params = { 
    "opt_name": [hp[0] for hp in hp_matrix],
    "l_rate": [hp[1] for hp in hp_matrix],
    "epsilon": [hp[2] for hp in hp_matrix],
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
control_params = {
    "model_name": model_name,
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 40ms/step - loss: 1005.7009 - f1_score: 0.8141 - accuracy: 0.6954 - precision_3: 0.6898 - recall_3: 0.9929
3/3 [==============================] - 0s 42ms/step - loss: 1355.4073 - f1_score: 0.7891 - accuracy: 0.6517 - precision_3: 0.6667 - recall_3: 0.9667
3/3 [==============================] - 0s 44ms/step - loss: 1336.3508 - f1_score: 0.7891 - accuracy: 0.6593 - precision_3: 0.6667 - recall_3: 0.9667
14/14 [==============================] - 1s 37ms/step
3/3 [==============================] - 0s 36ms/step
3/3 [==============================] - 0s 40ms/step

14/14 [==============================] - 1s 41ms/step - loss: 178.6911 - f1_score: 0.0000e+00 - accuracy: 0.3285 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00
3/3 [==============================] - 0s 36ms/step - loss: 188.4628 - f1_score: 0.0000e+00 - accuracy: 0.3258 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00
3/3 [==============================] - 0s 39ms/step - loss: 187.2834 - f1_score: 0.0000e+00 - accuracy: 0.3407 - precision_4: 0.0000e+00 - recall_4: 0.0000e+00
14/14 [==============================] - 1s 36ms/step
3/3 [==============================] - 0s 35ms/step
3/3 [==============================] - 0s 37ms/step

Best hyperparameters:

• Optimiser —> adam
• Learning rate —> 0.50438527
• Epsilon —> 0.05359655

best_optimiser = "adam"
best_l_rate = 0.50438527
best_epsilon = 0.05359655

Step 2: Adjusting epochs with newly tuned hyperparameters

# parameters that we want to iterate through
test_params = { 
    "epochs": [50]
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
    "model_name": model_name,
    "opt_name": best_optimiser,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 50ms/step - loss: 218.3432 - f1_score: 0.8034 - accuracy: 0.6715 - precision_7: 0.6715 - recall_7: 1.0000
3/3 [==============================] - 0s 51ms/step - loss: 217.8226 - f1_score: 0.8054 - accuracy: 0.6742 - precision_7: 0.6742 - recall_7: 1.0000
3/3 [==============================] - 0s 45ms/step - loss: 226.5419 - f1_score: 0.7947 - accuracy: 0.6593 - precision_7: 0.6593 - recall_7: 1.0000
14/14 [==============================] - 1s 43ms/step
3/3 [==============================] - 0s 72ms/step
3/3 [==============================] - 0s 39ms/step

Ideal epoch number: 15

model_hps[model_name] = {
    "epochs": 15,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon,
    "optimiser": best_optimiser
}
model_hps

{'perceptron': {'epochs': 20,
  'l_rate': 0.022246600000000002,
  'epsilon': 0.7611846600000001,
  'optimiser': 'adagrad'},
 'test_cnn_1': {'epochs': 15,
  'l_rate': 0.50438527,
  'epsilon': 0.05359655,
  'optimiser': 'adam'}}

Test Model 2

model_name = "test_cnn_2"

Step 1: Use a variety of search methods to determine the best learning rate, epsilon and optimiser

Calculating the best hyperparameters using random, hyperband and bayesian search

best_hps = {
    method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
    for method in hp_search_methods
}
best_hps

Trial 5 Complete [00h 00m 21s]
val_accuracy: 0.6741573214530945

Best val_accuracy So Far: 0.6741573214530945
Total elapsed time: 00h 01m 47s
INFO:tensorflow:Oracle triggered exit

{'random': {'optimiser': 'adam',
  'l_rate': 0.8048532700000001,
  'epsilon': 0.91542115},
 'bayesian': {'optimiser': 'sgd',
  'l_rate': 0.5362420800000001,
  'epsilon': 0.05256905}}

Comparing model performace with the hyperparameters yielded by the different search methods

# parameters that we want to iterate through
hp_matrix = [
    x
    for x in set(
        (
            best_hps[hp]["optimiser"], 
            best_hps[hp]["l_rate"],
            best_hps[hp]["epsilon"],
        ) 
        for hp in hp_search_methods
    )
]

test_params = { 
    "opt_name": [hp[0] for hp in hp_matrix],
    "l_rate": [hp[1] for hp in hp_matrix],
    "epsilon": [hp[2] for hp in hp_matrix],
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
control_params = {
    "model_name": model_name,
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 53ms/step - loss: 10.8099 - f1_score: 0.7993 - accuracy: 0.7410 - precision_1: 0.8333 - recall_1: 0.7679
3/3 [==============================] - 0s 54ms/step - loss: 17.0698 - f1_score: 0.6325 - accuracy: 0.5169 - precision_1: 0.6491 - recall_1: 0.6167
3/3 [==============================] - 0s 56ms/step - loss: 19.9935 - f1_score: 0.6218 - accuracy: 0.5055 - precision_1: 0.6271 - recall_1: 0.6167
14/14 [==============================] - 1s 53ms/step
3/3 [==============================] - 0s 52ms/step
3/3 [==============================] - 0s 55ms/step

14/14 [==============================] - 1s 53ms/step - loss: 104.0018 - f1_score: 0.8034 - accuracy: 0.6715 - precision_2: 0.6715 - recall_2: 1.0000
3/3 [==============================] - 0s 55ms/step - loss: 104.1037 - f1_score: 0.8054 - accuracy: 0.6742 - precision_2: 0.6742 - recall_2: 1.0000
3/3 [==============================] - 0s 55ms/step - loss: 107.7250 - f1_score: 0.7947 - accuracy: 0.6593 - precision_2: 0.6593 - recall_2: 1.0000
14/14 [==============================] - 1s 56ms/step
3/3 [==============================] - 0s 50ms/step
3/3 [==============================] - 0s 50ms/step

Best hyperparameters:

• Optimiser —> sgd
• Learning rate —> 0.5362420800000001
• Epsilon —> 0.05256905

best_optimiser = "sgd"
best_l_rate = 0.5362420800000001
best_epsilon = 0.05256905

Step 2: Adjusting epochs with newly tuned hyperparameters

# parameters that we want to iterate through
test_params = { 
    "epochs": [50]
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
    "model_name": model_name,
    "opt_name": best_optimiser,
    "l_rate": best_l_rate,
    "epsilon": best_l_rate
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 52ms/step - loss: 167.7011 - f1_score: 0.8034 - accuracy: 0.6715 - precision_3: 0.6715 - recall_3: 1.0000
3/3 [==============================] - 0s 52ms/step - loss: 167.3920 - f1_score: 0.8054 - accuracy: 0.6742 - precision_3: 0.6742 - recall_3: 1.0000
3/3 [==============================] - 0s 54ms/step - loss: 173.4474 - f1_score: 0.7947 - accuracy: 0.6593 - precision_3: 0.6593 - recall_3: 1.0000
14/14 [==============================] - 1s 52ms/step
3/3 [==============================] - 0s 49ms/step
3/3 [==============================] - 0s 51ms/step

Ideal epoch number: 30

model_hps[model_name] = {
    "epochs": 30,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon,
    "optimiser": best_optimiser
}
model_hps

{'perceptron': {'epochs': 20,
  'l_rate': 0.022246600000000002,
  'epsilon': 0.7611846600000001,
  'optimiser': 'adagrad'},
 'test_cnn_1': {'epochs': 15,
  'l_rate': 0.50438527,
  'epsilon': 0.05359655,
  'optimiser': 'adam'},
 'test_cnn_2': {'epochs': 30,
  'l_rate': 0.5362420800000001,
  'epsilon': 0.05256905,
  'optimiser': 'sgd'}}

Test Model 3

model_name = "test_cnn_3"

Step 1: Use a variety of search methods to determine the best learning rate, epsilon and optimiser

Calculating the best hyperparameters using random, hyperband and bayesian search

best_hps = {
    method: get_best_hyperparams(model_name, train_ds, val_ds, optimiser_range, method)
    for method in hp_search_methods
}
best_hps

Trial 5 Complete [00h 00m 24s]
val_accuracy: 0.6741573214530945

Best val_accuracy So Far: 0.7191011309623718
Total elapsed time: 00h 02m 01s
INFO:tensorflow:Oracle triggered exit

{'random': {'optimiser': 'nadam', 'l_rate': 0.15811354, 'epsilon': 0.06356881},
 'bayesian': {'optimiser': 'adam',
  'l_rate': 0.29507792,
  'epsilon': 0.35725662}}

Comparing model performace with the hyperparameters yielded by the different search methods

# parameters that we want to iterate through
hp_matrix = [
    x
    for x in set(
        (
            best_hps[hp]["optimiser"], 
            best_hps[hp]["l_rate"],
            best_hps[hp]["epsilon"],
        ) 
        for hp in hp_search_methods
    )
]

test_params = { 
    "opt_name": [hp[0] for hp in hp_matrix],
    "l_rate": [hp[1] for hp in hp_matrix],
    "epsilon": [hp[2] for hp in hp_matrix],
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
control_params = {
    "model_name": model_name,
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 61ms/step - loss: 67.6354 - f1_score: 0.8092 - accuracy: 0.6835 - precision_1: 0.6796 - recall_1: 1.0000
3/3 [==============================] - 0s 61ms/step - loss: 71.3541 - f1_score: 0.8054 - accuracy: 0.6742 - precision_1: 0.6742 - recall_1: 1.0000
3/3 [==============================] - 0s 62ms/step - loss: 75.6457 - f1_score: 0.7947 - accuracy: 0.6593 - precision_1: 0.6593 - recall_1: 1.0000
14/14 [==============================] - 1s 58ms/step
3/3 [==============================] - 0s 56ms/step
3/3 [==============================] - 0s 57ms/step

14/14 [==============================] - 1s 61ms/step - loss: 7.5362 - f1_score: 0.7269 - accuracy: 0.6379 - precision_2: 0.7363 - recall_2: 0.7179
3/3 [==============================] - 0s 62ms/step - loss: 9.4470 - f1_score: 0.6977 - accuracy: 0.5618 - precision_2: 0.6522 - recall_2: 0.7500
3/3 [==============================] - 0s 63ms/step - loss: 10.2503 - f1_score: 0.6230 - accuracy: 0.4945 - precision_2: 0.6129 - recall_2: 0.6333
14/14 [==============================] - 1s 59ms/step
3/3 [==============================] - 0s 60ms/step
3/3 [==============================] - 0s 60ms/step

Best hyperparameters:

• Optimiser —> nadamm
• Learning rate —> 0.15811354
• Epsilon —> 0.06356881

best_optimiser = "nadam"
best_l_rate = 0.15811354
best_epsilon = 0.06356881

Step 2: Adjusting epochs with newly tuned hyperparameters

# parameters that we want to iterate through
test_params = { 
    "epochs": [50]
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
# of the test parameters
control_params = {
    "model_name": model_name,
    "opt_name": best_optimiser,
    "l_rate": best_l_rate,
    "epsilon": best_l_rate
}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 1s 61ms/step - loss: 114.2310 - f1_score: 0.0000e+00 - accuracy: 0.3285 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00
3/3 [==============================] - 0s 63ms/step - loss: 118.4614 - f1_score: 0.0000e+00 - accuracy: 0.3258 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00
3/3 [==============================] - 0s 61ms/step - loss: 116.9338 - f1_score: 0.0000e+00 - accuracy: 0.3407 - precision_8: 0.0000e+00 - recall_8: 0.0000e+00
14/14 [==============================] - 1s 59ms/step
3/3 [==============================] - 0s 60ms/step
3/3 [==============================] - 0s 60ms/step

Ideal epoch number: 10

model_hps[model_name] = {
    "epochs": 10,
    "l_rate": best_l_rate,
    "epsilon": best_epsilon,
    "optimiser": best_optimiser
}
model_hps

{'perceptron': {'epochs': 20,
  'l_rate': 0.022246600000000002,
  'epsilon': 0.7611846600000001,
  'optimiser': 'adagrad'},
 'test_cnn_1': {'epochs': 15,
  'l_rate': 0.50438527,
  'epsilon': 0.05359655,
  'optimiser': 'adam'},
 'test_cnn_2': {'epochs': 30,
  'l_rate': 0.5362420800000001,
  'epsilon': 0.05256905,
  'optimiser': 'sgd'},
 'test_cnn_3': {'epochs': 10,
  'l_rate': 0.15811354,
  'epsilon': 0.06356881,
  'optimiser': 'nadam'}}

Comparison of Fine-Tuned Models

# parameters that we want to iterate through
keys = list(model_hps.keys())

test_params = {
    "model_name": keys,
    "opt_name": [model_hps[k]["optimiser"] for k in keys],
    "l_rate": [model_hps[k]["l_rate"] for k in keys],
    "epsilon": [model_hps[k]["epsilon"] for k in keys],
    "epochs": [model_hps[k]["epochs"] for k in keys],
}

# parameters that differ from the default values but 
# that we want to keep constant through each iteration
control_params = {}

compare_configs(
    test_params,
    model_name,
    train_ds,
    val_ds,
    test_ds,
    control_params,
)

14/14 [==============================] - 0s 5ms/step - loss: 0.5380 - f1_score: 0.8976 - accuracy: 0.8681 - precision_9: 0.9377 - recall_9: 0.8607
3/3 [==============================] - 0s 7ms/step - loss: 2.8787 - f1_score: 0.6726 - accuracy: 0.5843 - precision_9: 0.7170 - recall_9: 0.6333
3/3 [==============================] - 0s 7ms/step - loss: 3.3214 - f1_score: 0.6833 - accuracy: 0.5824 - precision_9: 0.6833 - recall_9: 0.6833
14/14 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 5ms/step
3/3 [==============================] - 0s 5ms/step

14/14 [==============================] - 1s 40ms/step - loss: 294.8774 - f1_score: 0.8834 - accuracy: 0.8537 - precision_10: 0.9506 - recall_10: 0.8250
3/3 [==============================] - 0s 42ms/step - loss: 1037.4180 - f1_score: 0.6372 - accuracy: 0.5393 - precision_10: 0.6792 - recall_10: 0.6000
3/3 [==============================] - 0s 43ms/step - loss: 1015.6523 - f1_score: 0.6724 - accuracy: 0.5824 - precision_10: 0.6964 - recall_10: 0.6500
14/14 [==============================] - 1s 39ms/step
3/3 [==============================] - 0s 38ms/step
3/3 [==============================] - 0s 39ms/step

14/14 [==============================] - 1s 54ms/step - loss: 64.5133 - f1_score: 0.4167 - accuracy: 0.4964 - precision_11: 0.9375 - recall_11: 0.2679
3/3 [==============================] - 0s 55ms/step - loss: 72.7705 - f1_score: 0.3467 - accuracy: 0.4494 - precision_11: 0.8667 - recall_11: 0.2167
3/3 [==============================] - 0s 55ms/step - loss: 73.6770 - f1_score: 0.2162 - accuracy: 0.3626 - precision_11: 0.5714 - recall_11: 0.1333
14/14 [==============================] - 1s 57ms/step
3/3 [==============================] - 0s 51ms/step
3/3 [==============================] - 0s 52ms/step

14/14 [==============================] - 1s 62ms/step - loss: 33.9206 - f1_score: 0.8070 - accuracy: 0.6811 - precision_12: 0.6797 - recall_12: 0.9929
3/3 [==============================] - 0s 71ms/step - loss: 35.1535 - f1_score: 0.7891 - accuracy: 0.6517 - precision_12: 0.6667 - recall_12: 0.9667
3/3 [==============================] - 0s 61ms/step - loss: 34.4937 - f1_score: 0.7919 - accuracy: 0.6593 - precision_12: 0.6629 - recall_12: 0.9833
14/14 [==============================] - 1s 57ms/step
3/3 [==============================] - 0s 55ms/step
3/3 [==============================] - 0s 59ms/step

Conclusion

Whilst the perceptron and single convolutional layer models display a similar level of performance, the 3-layer convolutional network has the clear edge both in terms of accuracy and F1 score.